{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T13:33:18.860125Z",
     "start_time": "2025-07-18T13:33:17.157743Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from tqdm import tqdm\n",
    "from peft import  LoraConfig, get_peft_model\n",
    "from modelscope import AutoTokenizer, AutoModel\n",
    "from torch.utils.data import DataLoader, Dataset"
   ],
   "id": "32234b3295b97474",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T13:33:28.746510Z",
     "start_time": "2025-07-18T13:33:21.863679Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model_dir = \"C:\\\\Users\\\\16014\\\\.cache\\\\modelscope\\\\hub\\\\models\\\\ZhipuAI\\\\chatglm3-6b\"\n",
    "with torch.no_grad():\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)\n",
    "    model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).half().cuda()"
   ],
   "id": "20408dba076fc19b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "d19032ebdabf470c8d3f5236b9204baa"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T13:33:30.493672Z",
     "start_time": "2025-07-18T13:33:30.487165Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lora_config = LoraConfig(\n",
    "    r=8,\n",
    "    lora_alpha=16,\n",
    "    target_modules=[\"query_key_value\"],#query_key_value\n",
    "    lora_dropout=0.05,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",  #SEQ_2_SEQ_LM\n",
    ")"
   ],
   "id": "45259f0f197f4fa1",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-18T13:34:02.307450Z",
     "start_time": "2025-07-18T13:34:02.218644Z"
    }
   },
   "cell_type": "code",
   "source": [
    "BATCH_SIZE = 1\n",
    "LEARNING_RATE = 2e-4\n",
    "device = \"cuda\"\n",
    "\n",
    "model = get_peft_model(model, lora_config)\n",
    "model.print_trainable_parameters()\n",
    "\n",
    "import get_data\n",
    "train_dataset = get_data.ChatDataset()\n",
    "datacollect = get_data.DataCollatorForChatDataset()\n",
    "train_loader = (DataLoader(train_dataset, batch_size=BATCH_SIZE,shuffle=True,collate_fn=datacollect))\n",
    "\n",
    "loss_fun = torch.nn.CrossEntropyLoss(ignore_index=-100)\n",
    "\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr = LEARNING_RATE)\n",
    "lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max = 2400,eta_min=2e-6,last_epoch=-1)"
   ],
   "id": "fc3835fb0913f374",
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'bitsandbytes'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mModuleNotFoundError\u001B[0m                       Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[5], line 5\u001B[0m\n\u001B[0;32m      2\u001B[0m LEARNING_RATE \u001B[38;5;241m=\u001B[39m \u001B[38;5;241m2e-4\u001B[39m\n\u001B[0;32m      3\u001B[0m device \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcuda\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m----> 5\u001B[0m model \u001B[38;5;241m=\u001B[39m \u001B[43mget_peft_model\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlora_config\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m      6\u001B[0m model\u001B[38;5;241m.\u001B[39mprint_trainable_parameters()\n\u001B[0;32m      8\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mget_data\u001B[39;00m\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\mapping.py:193\u001B[0m, in \u001B[0;36mget_peft_model\u001B[1;34m(model, peft_config, adapter_name, mixed, autocast_adapter_dtype, revision)\u001B[0m\n\u001B[0;32m    191\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m peft_config\u001B[38;5;241m.\u001B[39mis_prompt_learning:\n\u001B[0;32m    192\u001B[0m     peft_config \u001B[38;5;241m=\u001B[39m _prepare_prompt_learning_config(peft_config, model_config)\n\u001B[1;32m--> 193\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mMODEL_TYPE_TO_PEFT_MODEL_MAPPING\u001B[49m\u001B[43m[\u001B[49m\u001B[43mpeft_config\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtask_type\u001B[49m\u001B[43m]\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    194\u001B[0m \u001B[43m    \u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mpeft_config\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43madapter_name\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43madapter_name\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mautocast_adapter_dtype\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mautocast_adapter_dtype\u001B[49m\n\u001B[0;32m    195\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\peft_model.py:1609\u001B[0m, in \u001B[0;36mPeftModelForCausalLM.__init__\u001B[1;34m(self, model, peft_config, adapter_name, **kwargs)\u001B[0m\n\u001B[0;32m   1606\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21m__init__\u001B[39m(\n\u001B[0;32m   1607\u001B[0m     \u001B[38;5;28mself\u001B[39m, model: torch\u001B[38;5;241m.\u001B[39mnn\u001B[38;5;241m.\u001B[39mModule, peft_config: PeftConfig, adapter_name: \u001B[38;5;28mstr\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdefault\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs\n\u001B[0;32m   1608\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m-> 1609\u001B[0m     \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__init__\u001B[39m(model, peft_config, adapter_name, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m   1610\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbase_model_prepare_inputs_for_generation \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbase_model\u001B[38;5;241m.\u001B[39mprepare_inputs_for_generation\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\peft_model.py:171\u001B[0m, in \u001B[0;36mPeftModel.__init__\u001B[1;34m(self, model, peft_config, adapter_name, autocast_adapter_dtype, low_cpu_mem_usage)\u001B[0m\n\u001B[0;32m    169\u001B[0m     ctx \u001B[38;5;241m=\u001B[39m init_empty_weights \u001B[38;5;28;01mif\u001B[39;00m low_cpu_mem_usage \u001B[38;5;28;01melse\u001B[39;00m nullcontext\n\u001B[0;32m    170\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m ctx():\n\u001B[1;32m--> 171\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbase_model \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43m{\u001B[49m\u001B[43madapter_name\u001B[49m\u001B[43m:\u001B[49m\u001B[43m \u001B[49m\u001B[43mpeft_config\u001B[49m\u001B[43m}\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43madapter_name\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    172\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mset_additional_trainable_modules(peft_config, adapter_name)\n\u001B[0;32m    174\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mhasattr\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbase_model, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_cast_adapter_dtype\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\tuners\\lora\\model.py:141\u001B[0m, in \u001B[0;36mLoraModel.__init__\u001B[1;34m(self, model, config, adapter_name, low_cpu_mem_usage)\u001B[0m\n\u001B[0;32m    140\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21m__init__\u001B[39m(\u001B[38;5;28mself\u001B[39m, model, config, adapter_name, low_cpu_mem_usage: \u001B[38;5;28mbool\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m--> 141\u001B[0m     \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[38;5;21;43m__init__\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mconfig\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43madapter_name\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlow_cpu_mem_usage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlow_cpu_mem_usage\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\tuners\\tuners_utils.py:184\u001B[0m, in \u001B[0;36mBaseTuner.__init__\u001B[1;34m(self, model, peft_config, adapter_name, low_cpu_mem_usage)\u001B[0m\n\u001B[0;32m    182\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_pre_injection_hook(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodel, \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mpeft_config[adapter_name], adapter_name)\n\u001B[0;32m    183\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m peft_config \u001B[38;5;241m!=\u001B[39m PeftType\u001B[38;5;241m.\u001B[39mXLORA \u001B[38;5;129;01mor\u001B[39;00m peft_config[adapter_name] \u001B[38;5;241m!=\u001B[39m PeftType\u001B[38;5;241m.\u001B[39mXLORA:\n\u001B[1;32m--> 184\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minject_adapter\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43madapter_name\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mlow_cpu_mem_usage\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlow_cpu_mem_usage\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    186\u001B[0m \u001B[38;5;66;03m# Copy the peft_config in the injected model.\u001B[39;00m\n\u001B[0;32m    187\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodel\u001B[38;5;241m.\u001B[39mpeft_config \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mpeft_config\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\tuners\\tuners_utils.py:496\u001B[0m, in \u001B[0;36mBaseTuner.inject_adapter\u001B[1;34m(self, model, adapter_name, autocast_adapter_dtype, low_cpu_mem_usage)\u001B[0m\n\u001B[0;32m    494\u001B[0m     ctx \u001B[38;5;241m=\u001B[39m init_empty_weights \u001B[38;5;28;01mif\u001B[39;00m low_cpu_mem_usage \u001B[38;5;28;01melse\u001B[39;00m nullcontext\n\u001B[0;32m    495\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m ctx():\n\u001B[1;32m--> 496\u001B[0m         \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_create_and_replace\u001B[49m\u001B[43m(\u001B[49m\u001B[43mpeft_config\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43madapter_name\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtarget\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtarget_name\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mparent\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcurrent_key\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mkey\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    498\u001B[0m tied_target_modules \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_get_tied_target_modules(model\u001B[38;5;241m=\u001B[39mmodel)\n\u001B[0;32m    499\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m tied_target_modules:\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\tuners\\lora\\model.py:226\u001B[0m, in \u001B[0;36mLoraModel._create_and_replace\u001B[1;34m(self, lora_config, adapter_name, target, target_name, parent, current_key)\u001B[0m\n\u001B[0;32m    216\u001B[0m     target\u001B[38;5;241m.\u001B[39mupdate_layer(\n\u001B[0;32m    217\u001B[0m         adapter_name,\n\u001B[0;32m    218\u001B[0m         r,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    223\u001B[0m         use_dora\u001B[38;5;241m=\u001B[39mlora_config\u001B[38;5;241m.\u001B[39muse_dora,\n\u001B[0;32m    224\u001B[0m     )\n\u001B[0;32m    225\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m--> 226\u001B[0m     new_module \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_create_new_module(lora_config, adapter_name, target, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    227\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m adapter_name \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mactive_adapters:\n\u001B[0;32m    228\u001B[0m         \u001B[38;5;66;03m# adding an additional adapter: it is not automatically trainable\u001B[39;00m\n\u001B[0;32m    229\u001B[0m         new_module\u001B[38;5;241m.\u001B[39mrequires_grad_(\u001B[38;5;28;01mFalse\u001B[39;00m)\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\tuners\\lora\\model.py:321\u001B[0m, in \u001B[0;36mLoraModel._create_new_module\u001B[1;34m(lora_config, adapter_name, target, **kwargs)\u001B[0m\n\u001B[0;32m    319\u001B[0m \u001B[38;5;66;03m# avoid eager bnb import\u001B[39;00m\n\u001B[0;32m    320\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_bnb_available():\n\u001B[1;32m--> 321\u001B[0m     \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbnb\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m dispatch_bnb_8bit\n\u001B[0;32m    323\u001B[0m     dispatchers\u001B[38;5;241m.\u001B[39mappend(dispatch_bnb_8bit)\n\u001B[0;32m    325\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_bnb_4bit_available():\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\chatglm3-qlora\\lib\\site-packages\\peft\\tuners\\lora\\bnb.py:19\u001B[0m\n\u001B[0;32m     16\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mwarnings\u001B[39;00m\n\u001B[0;32m     17\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtyping\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Any, Optional\n\u001B[1;32m---> 19\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mbitsandbytes\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mbnb\u001B[39;00m\n\u001B[0;32m     20\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\n\u001B[0;32m     22\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mpeft\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mimport_utils\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m is_bnb_4bit_available, is_bnb_available\n",
      "\u001B[1;31mModuleNotFoundError\u001B[0m: No module named 'bitsandbytes'"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-04T21:58:25.901386Z",
     "start_time": "2025-07-04T21:58:03.862826Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for epoch in range(10):\n",
    "    pbar = tqdm(train_loader,total=len(train_loader))\n",
    "    for data_dict in pbar:\n",
    "        optimizer.zero_grad()\n",
    "        input_ids = data_dict[\"input_ids\"].to(device);input_ids = input_ids[:,:-1]\n",
    "        labels = data_dict[\"labels\"].to(device);labels = labels[:,1:]\n",
    "        logits = model(input_ids)[\"logits\"]\n",
    "        logits = logits.view(-1, logits.size(-1));labels = labels.view(-1)\n",
    "        loss = loss_fun(logits, labels)\n",
    "\n",
    "        # outputs = model(\n",
    "        #     input_ids=input_ids,\n",
    "        #     labels=labels,\n",
    "        # )\n",
    "        # loss = outputs.loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()  # 执行优化器\n",
    "\n",
    "\n",
    "        pbar.set_description(\n",
    "            f\"epoch:{epoch + 1}, train_loss:{loss.item():.5f}, lr:{lr_scheduler.get_last_lr()[0] * 1000:.5f}\")\n",
    "\n",
    "model.save_pretrained(\"./lora_saver/lora_query_key_value\")\n",
    "\n",
    "\n"
   ],
   "id": "5d99dc4cfa047c47",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch:1, train_loss:0.22754, lr:0.19994: 100%|██████████| 27/27 [00:03<00:00,  8.99it/s]\n",
      "epoch:2, train_loss:0.00291, lr:0.19975: 100%|██████████| 27/27 [00:02<00:00, 13.07it/s]\n",
      "epoch:3, train_loss:0.00069, lr:0.19944: 100%|██████████| 27/27 [00:02<00:00, 13.07it/s]\n",
      "epoch:4, train_loss:0.00068, lr:0.19901: 100%|██████████| 27/27 [00:02<00:00, 12.65it/s]\n",
      "epoch:5, train_loss:0.00059, lr:0.19846: 100%|██████████| 27/27 [00:02<00:00, 12.75it/s]\n",
      "epoch:6, train_loss:0.00064, lr:0.19778: 100%|██████████| 27/27 [00:02<00:00, 12.60it/s]\n",
      "epoch:7, train_loss:0.00057, lr:0.19699: 100%|██████████| 27/27 [00:02<00:00, 12.64it/s]\n",
      "epoch:8, train_loss:0.00091, lr:0.19607: 100%|██████████| 27/27 [00:02<00:00, 12.66it/s]\n",
      "epoch:9, train_loss:0.00087, lr:0.19503: 100%|██████████| 27/27 [00:02<00:00, 12.68it/s]\n",
      "epoch:10, train_loss:0.00103, lr:0.19388: 100%|██████████| 27/27 [00:02<00:00, 13.05it/s]\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "ce7c4a0204213af6"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
